# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo

import json

class MongoPipeline(object):
    """docstring for MongoPipline"""

    def __init__(self, MONGOD_URL, MONGOD_DB, MONGOD_TABLE):
        
        self.MONGOD_URL = MONGOD_URL
        self.MONGOD_DB = MONGOD_DB
        self.MONGOD_TABLE = MONGOD_TABLE

        


    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            MONGOD_URL=crawler.settings.get('MONGOD_URL'),
            MONGOD_TABLE = crawler.settings.get('MONGOD_TABLE'),
            MONGOD_DB = crawler.settings.get('MONGOD_DB', 'items')
            )

    def open_spider(self,spider):
        self.client = pymongo.MongoClient(self.MONGOD_URL)
        self.db = self.client[self.MONGOD_DB]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        old_data = [item['url'] for item in self.db[self.MONGOD_TABLE].find()]
        if item["url"] in old_data:
            print('数据已经存在')
            return None
        if self.db[self.MONGOD_TABLE].insert(dict(item)):
            print('成功插入一条数据')
        return item

